# install.packages("pacman")
pacman::p_load(tidyverse,fastDummies,data.table)
#function
zero1 <- function(x, minx=NA, maxx=NA){
  res <- NA
  if(is.na(minx)) res <- (x - min(x,na.rm=T))/(max(x,na.rm=T) -min(x,na.rm=T))
  if(!is.na(minx)) res <- (x - minx)/(maxx -minx)
  res
}

## Note: Data must be downloaded from LISS
# Personality - load data----------
pers01<-rio::import("Data/cp08a_1p_EN.dta")
pers06<-rio::import("Data/cp13f_EN_1.0p.dta")
pers11<-rio::import("Data/cp19k_EN_1.0p.dta")

# Personality 2008 -----------------------------------------------------------
#Openness
pers01$open8_1<- pers01$cp08a024
pers01$open8_2rec<- 6-pers01$cp08a029
pers01$open8_3<- pers01$cp08a034
pers01$open8_4rec<- 6-pers01$cp08a039
pers01$open8_5<- pers01$cp08a044
pers01$open8_6rec<- 6-pers01$cp08a049
pers01$open8_7<- pers01$cp08a054
pers01$open8_8<- pers01$cp08a059
pers01$open8_9<- pers01$cp08a064
pers01$open8_10<- pers01$cp08a069

#Conscientiousness
pers01$con8_1<- pers01$cp08a022
pers01$con8_2rec<- 6-pers01$cp08a027
pers01$con8_3<- pers01$cp08a032
pers01$con8_4rec<- 6-pers01$cp08a037
pers01$con8_5<- pers01$cp08a042
pers01$con8_6rec<- 6-pers01$cp08a047
pers01$con8_7<- pers01$cp08a052
pers01$con8_8rec<- 6-pers01$cp08a057
pers01$con8_9<- pers01$cp08a062
pers01$con8_10<- pers01$cp08a067

#Extraversion
pers01$ext8_1<- pers01$cp08a020
pers01$ext8_2rec<- 6-pers01$cp08a025
pers01$ext8_3<- pers01$cp08a030
pers01$ext8_4rec<- 6-pers01$cp08a035
pers01$ext8_5<- pers01$cp08a040
pers01$ext8_6rec<- 6-pers01$cp08a045
pers01$ext8_7<- pers01$cp08a050
pers01$ext8_8rec<- 6-pers01$cp08a055
pers01$ext8_9<- pers01$cp08a060
pers01$ext8_10rec<- 6-pers01$cp08a065

#Agreeableness
pers01$agre8_1rec<- 6-pers01$cp08a021
pers01$agre8_2<- pers01$cp08a026
pers01$agre8_3rec<- 6-pers01$cp08a031
pers01$agre8_4<- pers01$cp08a036
pers01$agre8_5rec<- 6-pers01$cp08a041
pers01$agre8_6<- pers01$cp08a046
pers01$agre8_7rec<- 6-pers01$cp08a051
pers01$agre8_8<- pers01$cp08a056
pers01$agre8_9<- pers01$cp08a061
pers01$agre8_10<- pers01$cp08a066

#Neuroticism
pers01$neu8_1<- pers01$cp08a023
pers01$neu8_2rec<- 6-pers01$cp08a028
pers01$neu8_3<- pers01$cp08a033
pers01$neu8_4rec<- 6-pers01$cp08a038
pers01$neu8_5<- pers01$cp08a043
pers01$neu8_6<- pers01$cp08a048
pers01$neu8_7<- pers01$cp08a053
pers01$neu8_8<- pers01$cp08a058
pers01$neu8_9<- pers01$cp08a063
pers01$neu8_10<- pers01$cp08a068


open <-data.frame(pers01$open8_1, pers01$open8_2rec, pers01$open8_3, pers01$open8_4rec, pers01$open8_5, pers01$open8_6rec, pers01$open8_7, pers01$open8_8, pers01$open8_9, pers01$open8_10)
pers01$open1_latent<-rowMeans(open, na.rm=T)

con8 <-data.frame(pers01$con8_1, pers01$con8_2rec, pers01$con8_3, pers01$con8_4rec, pers01$con8_5, pers01$con8_6rec, pers01$con8_7, pers01$con8_8rec, pers01$con8_9, pers01$con8_10)
pers01$con1_latent <- rowMeans(con8,na.rm=T)

ext8 <-data.frame(with(pers01, data.frame(ext8_1, ext8_2rec, ext8_3, ext8_4rec, ext8_5, ext8_6rec, ext8_7, ext8_8rec, ext8_9, ext8_10rec)))
pers01$ext1_latent<-rowMeans(ext8, na.rm=T)

agre8 <- data.frame(with(pers01, data.frame(agre8_1rec, agre8_2, agre8_3rec, agre8_4, agre8_5rec, agre8_6, agre8_7rec, agre8_8, agre8_9, agre8_10)))
pers01$agre1_latent<-rowMeans(agre8, na.rm=T)

neu8<-data.frame(with(pers01, data.frame(neu8_1, neu8_2rec, neu8_3, neu8_4rec, neu8_5, neu8_6, neu8_7, neu8_8, neu8_9, neu8_10)))
pers01$neu1_latent<-rowMeans(neu8, na.rm=T)
#select variables
pers01 %>% dplyr::select(ends_with("latent"),nomem_encr) -> pers01_short

# Personality 2013 -----------------------------------------------------------
pers06$open8_1<- pers06$cp13f024
pers06$open8_2rec<- 6-pers06$cp13f029
pers06$open8_3<- pers06$cp13f034
pers06$open8_4rec<- 6-pers06$cp13f039
pers06$open8_5<- pers06$cp13f044
pers06$open8_6rec<- 6-pers06$cp13f049
pers06$open8_7<- pers06$cp13f054
pers06$open8_8<- pers06$cp13f059
pers06$open8_9<- pers06$cp13f064
pers06$open8_10<- pers06$cp13f069

#Conscientiousness
pers06$con8_1<- pers06$cp13f022
pers06$con8_2rec<- 6-pers06$cp13f027
pers06$con8_3<- pers06$cp13f032
pers06$con8_4rec<- 6-pers06$cp13f037
pers06$con8_5<- pers06$cp13f042
pers06$con8_6rec<- 6-pers06$cp13f047
pers06$con8_7<- pers06$cp13f052
pers06$con8_8rec<- 6-pers06$cp13f057
pers06$con8_9<- pers06$cp13f062
pers06$con8_10<- pers06$cp13f067

#Extraversion
pers06$ext8_1<- pers06$cp13f020
pers06$ext8_2rec<- 6-pers06$cp13f025
pers06$ext8_3<- pers06$cp13f030
pers06$ext8_4rec<- 6-pers06$cp13f035
pers06$ext8_5<- pers06$cp13f040
pers06$ext8_6rec<- 6-pers06$cp13f045
pers06$ext8_7<- pers06$cp13f050
pers06$ext8_8rec<- 6-pers06$cp13f055
pers06$ext8_9<- pers06$cp13f060
pers06$ext8_10rec<- 6-pers06$cp13f065

#Agreeableness
pers06$agre8_1rec<- 6-pers06$cp13f021
pers06$agre8_2<- pers06$cp13f026
pers06$agre8_3rec<- 6-pers06$cp13f031
pers06$agre8_4<- pers06$cp13f036
pers06$agre8_5rec<- 6-pers06$cp13f041
pers06$agre8_6<- pers06$cp13f046
pers06$agre8_7rec<- 6-pers06$cp13f051
pers06$agre8_8<- pers06$cp13f056
pers06$agre8_9<- pers06$cp13f061
pers06$agre8_10<- pers06$cp13f066

#Neuroticism
pers06$neu8_1<- pers06$cp13f023
pers06$neu8_2rec<- 6-pers06$cp13f028
pers06$neu8_3<- pers06$cp13f033
pers06$neu8_4rec<- 6-pers06$cp13f038
pers06$neu8_5<- pers06$cp13f043
pers06$neu8_6<- pers06$cp13f048
pers06$neu8_7<- pers06$cp13f053
pers06$neu8_8<- pers06$cp13f058
pers06$neu8_9<- pers06$cp13f063
pers06$neu8_10<- pers06$cp13f068

open <-data.frame(pers06$open8_1, pers06$open8_2rec, pers06$open8_3, pers06$open8_4rec, pers06$open8_5, pers06$open8_6rec, pers06$open8_7, pers06$open8_8, pers06$open8_9, pers06$open8_10)
pers06$open6_latent<-rowMeans(open, na.rm=T)

con8 <-data.frame(pers06$con8_1, pers06$con8_2rec, pers06$con8_3, pers06$con8_4rec, pers06$con8_5, pers06$con8_6rec, pers06$con8_7, pers06$con8_8rec, pers06$con8_9, pers06$con8_10)
pers06$con6_latent <- rowMeans(con8,na.rm=T)

ext8 <-data.frame(with(pers06, data.frame(ext8_1, ext8_2rec, ext8_3, ext8_4rec, ext8_5, ext8_6rec, ext8_7, ext8_8rec, ext8_9, ext8_10rec)))
pers06$ext6_latent<-rowMeans(ext8, na.rm=T)

agre8 <- data.frame(with(pers06, data.frame(agre8_1rec, agre8_2, agre8_3rec, agre8_4, agre8_5rec, agre8_6, agre8_7rec, agre8_8, agre8_9, agre8_10)))
pers06$agre6_latent<-rowMeans(agre8, na.rm=T)

neu8<-data.frame(with(pers06, data.frame(neu8_1, neu8_2rec, neu8_3, neu8_4rec, neu8_5, neu8_6, neu8_7, neu8_8, neu8_9, neu8_10)))
pers06$neu6_latent<-rowMeans(neu8, na.rm=T)
#select variables
pers06 %>% dplyr::select(ends_with("latent"),nomem_encr) -> pers06_short

# Personality 2019 ----------------------------------------------------------
pers11$open8_1<- pers11$cp19k024
pers11$open8_2rec<- 6-pers11$cp19k029
pers11$open8_3<- pers11$cp19k034
pers11$open8_4rec<- 6-pers11$cp19k039
pers11$open8_5<- pers11$cp19k044
pers11$open8_6rec<- 6-pers11$cp19k049
pers11$open8_7<- pers11$cp19k054
pers11$open8_8<- pers11$cp19k059
pers11$open8_9<- pers11$cp19k064
pers11$open8_10<- pers11$cp19k069

#Conscientiousness
pers11$con8_1<- pers11$cp19k022
pers11$con8_2rec<- 6-pers11$cp19k027
pers11$con8_3<- pers11$cp19k032
pers11$con8_4rec<- 6-pers11$cp19k037
pers11$con8_5<- pers11$cp19k042
pers11$con8_6rec<- 6-pers11$cp19k047
pers11$con8_7<- pers11$cp19k052
pers11$con8_8rec<- 6-pers11$cp19k057
pers11$con8_9<- pers11$cp19k062
pers11$con8_10<- pers11$cp19k067

#Extraversion
pers11$ext8_1<- pers11$cp19k020
pers11$ext8_2rec<- 6-pers11$cp19k025
pers11$ext8_3<- pers11$cp19k030
pers11$ext8_4rec<- 6-pers11$cp19k035
pers11$ext8_5<- pers11$cp19k040
pers11$ext8_6rec<- 6-pers11$cp19k045
pers11$ext8_7<- pers11$cp19k050
pers11$ext8_8rec<- 6-pers11$cp19k055
pers11$ext8_9<- pers11$cp19k060
pers11$ext8_10rec<- 6-pers11$cp19k065

#Agreeableness
pers11$agre8_1rec<- 6-pers11$cp19k021
pers11$agre8_2<- pers11$cp19k026
pers11$agre8_3rec<- 6-pers11$cp19k031
pers11$agre8_4<- pers11$cp19k036
pers11$agre8_5rec<- 6-pers11$cp19k041
pers11$agre8_6<- pers11$cp19k046
pers11$agre8_7rec<- 6-pers11$cp19k051
pers11$agre8_8<- pers11$cp19k056
pers11$agre8_9<- pers11$cp19k061
pers11$agre8_10<- pers11$cp19k066

#Neuroticism
pers11$neu8_1<- pers11$cp19k023
pers11$neu8_2rec<- 6-pers11$cp19k028
pers11$neu8_3<- pers11$cp19k033
pers11$neu8_4rec<- 6-pers11$cp19k038
pers11$neu8_5<- pers11$cp19k043
pers11$neu8_6<- pers11$cp19k048
pers11$neu8_7<- pers11$cp19k053
pers11$neu8_8<- pers11$cp19k058
pers11$neu8_9<- pers11$cp19k063
pers11$neu8_10<- pers11$cp19k068


open <-data.frame(pers11$open8_1, pers11$open8_2rec, pers11$open8_3, pers11$open8_4rec, pers11$open8_5, pers11$open8_6rec, pers11$open8_7, pers11$open8_8, pers11$open8_9, pers11$open8_10)
pers11$open11_latent<-rowMeans(open, na.rm=T)

con8 <-data.frame(pers11$con8_1, pers11$con8_2rec, pers11$con8_3, pers11$con8_4rec, pers11$con8_5, pers11$con8_6rec, pers11$con8_7, pers11$con8_8rec, pers11$con8_9, pers11$con8_10)
pers11$con11_latent <- rowMeans(con8,na.rm=T)

ext8 <-data.frame(with(pers11, data.frame(ext8_1, ext8_2rec, ext8_3, ext8_4rec, ext8_5, ext8_6rec, ext8_7, ext8_8rec, ext8_9, ext8_10rec)))
pers11$ext11_latent<-rowMeans(ext8, na.rm=T)


agre8 <- data.frame(with(pers11, data.frame(agre8_1rec, agre8_2, agre8_3rec, agre8_4, agre8_5rec, agre8_6, agre8_7rec, agre8_8, agre8_9, agre8_10)))
pers11$agre11_latent<-rowMeans(agre8, na.rm=T)


neu8<-data.frame(with(pers11, data.frame(neu8_1, neu8_2rec, neu8_3, neu8_4rec, neu8_5, neu8_6, neu8_7, neu8_8, neu8_9, neu8_10)))
pers11$neu11_latent<-rowMeans(neu8, na.rm=T)

pers11 %>% dplyr::select(ends_with("latent"),nomem_encr) -> pers11_short

# Personality create personality data------------
list(pers01_short, pers06_short, pers11_short) %>% reduce(full_join, by = "nomem_encr") -> allpersonality

# Ideology - load data
pol01<-rio::import("Data/cv08a_1.1p_EN.sav")
pol06<-rio::import("Data/cv13f_EN_1.0p.sav")
pol11<-rio::import("Data/cv19k_EN_1.0p.sav")

# Ideology - Left-Right --------------------------------------------------------------

pol01$cv08a101[pol01$cv08a101==999]=NA
pol01$libcon01 <- pol01$cv08a101

pol06$cv13f101[pol06$cv13f101==999]=NA
pol06$libcon06 <- pol06$cv13f101

pol11$cv19k101[pol11$cv19k101==999]=NA
pol11$libcon11 <- pol11$cv19k101


# Ideology - economic ideology --------------------------------------------------------------

pol01$cv08a103[pol01$cv08a103==99]=NA
pol01$incomediffs01 <- pol01$cv08a103

pol06$cv13f103[pol06$cv13f103==99]=NA
pol06$incomediffs06 <- pol06$cv13f103

pol11$cv19k103[pol11$cv19k103==99]=NA
pol11$incomediffs11 <- pol11$cv19k103

# Ideology - Euthanasia ------------------------
pol01$cv08a102[pol01$cv08a102==99]=NA
pol01$euth01 <- 6- pol01$cv08a102

pol06$cv13f102[pol06$cv13f102==99]=NA
pol06$euth06 <- 6- pol06$cv13f102

pol11$cv19k102[pol11$cv19k102==99]=NA
pol11$euth11 <- 6- pol11$cv19k102


# Ideology - immigration -------------------------------------------------------------
pol01$cv08a104[pol01$cv08a104==99]=NA
pol01$immi01 <-  pol01$cv08a104

pol06$cv13f104[pol06$cv13f104==99]=NA
pol06$immi06 <-  pol06$cv13f104

pol11$cv19k104[pol11$cv19k104==99]=NA
pol11$immi11 <-  pol11$cv19k104


# Ideology - EU -------------------------------------------------------------
pol01$cv08a105[pol01$cv08a105==99]=NA
pol01$eu01 <-  pol01$cv08a105

pol06$cv13f105[pol06$cv13f105==99]=NA
pol06$eu06 <-  pol06$cv13f105

pol11$cv19k105[pol11$cv19k105==99]=NA
pol11$eu11 <-  pol11$cv19k105

# Ideology select variables ----------------
pol01_short <- dplyr::select(pol01,contains("libcon"),contains("incomediffs"),contains("euth"),contains("eu"),contains("immi"),nomem_encr)
pol06_short <- dplyr::select(pol06,contains("libcon"),contains("incomediffs"),contains("euth"),contains("eu"),contains("immi"),nomem_encr)
pol11_short <- dplyr::select(pol11,contains("libcon"),contains("incomediffs"),contains("euth"),contains("eu"),contains("immi"),nomem_encr)
list(pol01_short,pol06_short, pol11_short) %>% reduce(full_join, by = "nomem_encr") -> allpolitics

# Combine politics and persoanlity data --------
all <- merge(allpolitics,allpersonality,by="nomem_encr")

# Background - load data ----------------
avars1 <- rio::import("Data/avars_200805_EN_2.0p.sav")
avars2 <- rio::import("Data/avars_201305_EN_1.0p.sav")
avars3 <- rio::import("Data/avars_201911_EN_1.0p.sav")
avarsa <- merge(avars1,avars2,by="nomem_encr",all = T)
avarsa <- merge(avarsa,avars3,by="nomem_encr",all = T)

# Background - code demographic variables and select data------------
avarsa$gender <- avarsa$geslacht.x
avarsa$gender[is.na(avarsa$gender)] <- avarsa$geslacht.y[is.na(avarsa$gender)]
avarsa$gender[is.na(avarsa$gender)] <- avarsa$geslacht[is.na(avarsa$gender)]

avarsa$education <- avarsa$oplcat.x
avarsa$education[is.na(avarsa$education)] <- avarsa$oplcat.y[is.na(avarsa$education)]
avarsa$education[is.na(avarsa$education)] <- avarsa$oplcat[is.na(avarsa$education)]

avarsa$age <- avarsa$leeftijd.x
avarsa$age[is.na(avarsa$age)] <- avarsa$leeftijd.y[is.na(avarsa$age)]
avarsa$age[is.na(avarsa$age)] <- avarsa$leeftijd[is.na(avarsa$age)]
avarsa$income<- avarsa$nettohh_f.x
avarsa$income[is.na(avarsa$income)] <- avarsa$nettohh_f.y[is.na(avarsa$income)]

demos <- select(avarsa,age,education,gender, income, nomem_encr)
all <- merge(all, demos, by="nomem_encr",all.x = T)

# Combined data - code variables and create a long dataset-----------
#closed personality
all$open1_latent <- zero1(6 -all$open1_latent)
all$open6_latent <- zero1(6 -all$open6_latent)
all$open11_latent <- zero1(6 -all$open11_latent)

all$con1_latent <- zero1(all$con1_latent)
all$con6_latent <- zero1(all$con6_latent)
all$con11_latent <- zero1(all$con11_latent)

all$libcon01 <- zero1(all$libcon01)
all$libcon06 <- zero1(all$libcon06)
all$libcon11 <- zero1(all$libcon11)

all$incomediffs01 <- 1-zero1(all$incomediffs01)
all$incomediffs06 <- 1-zero1(all$incomediffs06)
all$incomediffs11 <- 1-zero1(all$incomediffs11)


all$immi01 <- zero1(all$immi01)
all$immi06 <- zero1(all$immi06)
all$immi11 <- zero1(all$immi11)

all$euth01 <- zero1(all$euth01)
all$euth06 <- zero1(all$euth06)
all$euth11 <- zero1(all$euth11)

all$eu01 <- zero1(all$eu01)
all$eu06 <- zero1(all$eu06)
all$eu11 <- zero1(all$eu11)

all$global01 <- rowMeans(with(all,data.frame(eu01,euth01,libcon01,incomediffs01,immi01)),na.rm=T)
all$global06 <- rowMeans(with(all,data.frame(eu06,euth06,libcon06,incomediffs06,immi06)),na.rm=T)
all$global11 <- rowMeans(with(all,data.frame(eu11,euth11,libcon11,incomediffs11,immi11)),na.rm=T)

long <- data.table::melt(data.table(all),measure.vars=list(c("open1_latent","con1_latent"),c("open6_latent","con6_latent"),c("open11_latent","con11_latent")),value.name=c("p01","p06","p11"))

#create education dummies
long$education[is.na(long$education)]=0
long <- fastDummies::dummy_cols(long,select_columns = "education")
#long <- dummy_cols(long,select_columns = "education")
long$variable <- as.numeric(long$variable)
# CLPM

#create income dummies
long$income4 <- as.numeric(gtools::quantcut(long$income,q = seq(0, 1, by=0.25),na.rm = T))
long$income4[is.na(long$income4)]=0
long <- fastDummies::dummy_cols(long,select_columns = "income4")

#save data
save(all,long,file = "Data/liss.RData")
